package parse_and_index.pdf;

import parse_and_index.build.inverted.Stemmer;
import parse_and_index.build.inverted.Tokenizator;

public class TokenizatorPDF extends Tokenizator implements java.io.Serializable {

	  public TokenizatorPDF(final String fn, final String ce, final long id) {
			super(fn, ce, id);
	  }

	  private static final long serialVersionUID = 1L;

	  @Override
	  protected String stemAndCheckToken(final String token) {
			final Stemmer s = new Stemmer(token);
			if (currentEncoding.equalsIgnoreCase("cp1251")) {
				  s.UTF8CyrillicToLatin();
			}
			return s.getTerm();
	  }
}